#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :read_douban.py
# @Time      :2023/9/8 
# @Author    :CL
# @email     :1037654919@qq.com
import time
import json
from utils import MongoDBUtil, mongo_manager
MongoDBUtil = MongoDBUtil(db_name="wx_read")
url  = 'https://read.douban.com/category/1?sort=hot&page=1'
pages =  3043

def retry(func):
    def wrapper(*args, **kwargs):
        for i in range(5):
            try:
                return func(*args, **kwargs)
            except Exception as e:
                print(e)
                time.sleep(1)
    return wrapper

@retry
def get_douban_book(kind =1,page =1 ):
    import requests
    import json

    headers = {
        "Accept": "application/json",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Connection": "keep-alive",
        "Content-Type": "application/json",
        "Origin": "https://read.douban.com",
        "Referer": "https://read.douban.com",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
        "X-CSRF-Token": "null",
        "X-Requested-With": "XMLHttpRequest",
        "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "bid": "IoWM2l63OaQ",
        "__utma": "30149280.1675583776.1693212024.1693212024.1693212024.1",
        "__utmz": "30149280.1693212024.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic",
        "ll": "\"118173\"",
        "_pk_id.100001.a7dd": "2ccc35dfbc4ca194.1694141612.",
        "_pk_ses.100001.a7dd": "1",
        "_ga": "GA1.1.1675583776.1693212024",
        "_gid": "GA1.3.1059874160.1694141613",
        "_ga_RXNMP372GL": "GS1.1.1694141613.1.1.1694141792.2.0.0"
    }
    url = "https://read.douban.com/j/kind/"
    data = {
        "sort": "hot",
        "page": page,
        "kind": kind,
        "query": "\n    query getFilterWorksList($works_ids: [ID!]) {\n      worksList(worksIds: $works_ids) {\n        \n    \n    title\n    cover(useSmall: false)\n    url\n    isBundle\n    coverLabel(preferVip: true)\n  \n    \n  url\n  title\n\n    \n  author {\n    name\n    url\n  }\n  origAuthor {\n    name\n    url\n  }\n  translator {\n    name\n    url\n  }\n\n    \n  abstract\n  authorHighlight\n  editorHighlight\n\n    \n    isOrigin\n    kinds {\n      \n    name @skip(if: true)\n    shortName @include(if: true)\n    id\n  \n    }\n    ... on WorksBase @include(if: true) {\n      wordCount\n      wordCountUnit\n    }\n    ... on WorksBase @include(if: false) {\n      inLibraryCount\n    }\n    ... on WorksBase @include(if: false) {\n      \n    isEssay\n    \n    ... on EssayWorks {\n      favorCount\n    }\n  \n    \n    \n    averageRating\n    ratingCount\n    url\n    isColumn\n    isFinished\n  \n  \n  \n    }\n    ... on EbookWorks @include(if: false) {\n      \n    ... on EbookWorks {\n      book {\n        url\n        averageRating\n        ratingCount\n      }\n    }\n  \n    }\n    ... on WorksBase @include(if: false) {\n      isColumn\n      isEssay\n      onSaleTime\n      ... on ColumnWorks {\n        updateTime\n      }\n    }\n    ... on WorksBase @include(if: true) {\n      isColumn\n      ... on ColumnWorks {\n        isFinished\n      }\n    }\n    ... on EssayWorks {\n      essayActivityData {\n        \n    title\n    uri\n    tag {\n      name\n      color\n      background\n      icon2x\n      icon3x\n      iconSize {\n        height\n      }\n      iconPosition {\n        x y\n      }\n    }\n  \n      }\n    }\n    highlightTags {\n      name\n    }\n    ... on WorksBase @include(if: false) {\n      fanfiction {\n        tags {\n          id\n          name\n          url\n        }\n      }\n    }\n  \n    \n  ... on WorksBase {\n    copyrightInfo {\n      newlyAdapted\n      newlyPublished\n      adaptedName\n      publishedName\n    }\n  }\n\n    isInLibrary\n    ... on WorksBase @include(if: false) {\n      \n    fixedPrice\n    salesPrice\n    isRebate\n  \n    }\n    ... on EbookWorks {\n      \n    fixedPrice\n    salesPrice\n    isRebate\n  \n    }\n    ... on WorksBase @include(if: true) {\n      ... on EbookWorks {\n        id\n        isPurchased\n        isInWishlist\n      }\n    }\n    ... on WorksBase @include(if: false) {\n      fanfiction {\n        fandoms {\n          title\n          url\n        }\n      }\n    }\n    ... on WorksBase @include(if: false) {\n      fanfiction {\n        kudoCount\n      }\n    }\n  \n        id\n        isOrigin\n        isEssay\n      }\n    }\n  ",
        "variables": {}
    }
    data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, cookies=cookies, data=data)

    # print(response.text)
    print(response)
    json_data = json.loads(response.text)
    lists = json_data['list']
    return lists

def parse_data(page):
    douban_books = mongo_manager("douban_books", db="wx_read")
    lists = get_douban_book(page = page)

    for item in lists:
        item['_id'] = item['id']
        try:
            MongoDBUtil.insert_one('douban_books', item)
        except Exception as e:
            print(e)
        finally:
            douban_books.updateOne({'_id': item['_id']}, item)
    douban_books.close()
    return
def main():

    for page  in  range(1, pages+1):
        print('-'*10,'begin',page,'-'*10)
        parse_data(page =page)
        if page % 10 == 0:
            print('time.sleep(5)')
            time.sleep(5)


    return

if __name__ == '__main__':
    print("start")
    main()